Google Cloud の Batch を使ってみる

# サービスアカウントを作成
$ gcloud iam service-accounts create sa-batch

# roles/batch.agentReporter の権限を付与
$ gcloud projects add-iam-policy-binding <PROJECT_ID> \
    --member="serviceAccount:sa-batch@<PROJECT_ID>.iam.gserviceaccount.com" \
    --role="roles/batch.agentReporter" \
    --condition None

# roles/storage.admin の権限を付与
$ gcloud projects add-iam-policy-binding <PROJECT_ID> \
    --member="serviceAccount:sa-batch@<PROJECT_ID>.iam.gserviceaccount.com" \
    --role="roles/storage.admin" \
    --condition None

# roles/logging.logWriter の権限を付与
$ gcloud projects add-iam-policy-binding <PROJECT_ID> \
    --member="serviceAccount:sa-batch@<PROJECT_ID>.iam.gserviceaccount.com" \
    --role="roles/logging.logWriter" \
    --condition None

# 圧縮されたファイルを保管するバケット
$ gcloud storage buckets create gs://hanzawa-yuya-compressed-files \
    --location=asia-northeast1 \
    --uniform-bucket-level-access

# 解凍したファイルを保管するバケット
$ gcloud storage buckets create gs://hanzawa-yuya-uncompressed-files \
    --location=asia-northeast1 \
    --uniform-bucket-level-access

$ echo "id,name" > sample.csv && echo "1,apple" >> sample.csv
$ gzip sample.csv
$ gcloud storage cp ./sample.csv.gz gs://hanzawa-yuya-compressed-files/test/

#!/bin/bash
echo "----"
echo $FILE_PATH
echo "----"
set -e

FILE_NAME=$(basename "$FILE_PATH")
FILE_DIR=$(dirname "$FILE_PATH")

echo "Downloading $FILE_NAME from $INPUT_BUCKET"
gcloud storage cp gs://$INPUT_BUCKET/$FILE_PATH ./tmp/

echo "Decompressing $FILE_NAME..."
gzip -d ./tmp/$FILE_NAME

BASE_NAME="${FILE_NAME%.*}"

# 解凍後のファイル名
DECOMPRESSED_FILE="./tmp/${BASE_NAME}"

echo "Uploading $BASE_NAME to $OUTPUT_BUCKET/$FILE_DIR"
gcloud storage cp $DECOMPRESSED_FILE gs://$OUTPUT_BUCKET/$FILE_DIR/

echo "Process completed successfully."

{
    "taskGroups": [
        {
            "taskSpec": {
                "computeResource": {
                "cpuMilli": "500",
                "memoryMib": "500"
                },
                "environment": {
                    "variables": {
                        "FILE_PATH": "test/sample.csv.gz",
                        "INPUT_BUCKET": "hanzawa-yuya-compressed-files",
                        "OUTPUT_BUCKET": "hanzawa-yuya-uncompressed-files"
                    }
                }
            }
        }
    ],
    "allocationPolicy": {
        "location": {
            "allowedLocations": ["regions/asia-northeast1"]
        },
        "instances": [
            {
                "policy": {
                    "machineType": "e2-micro",
                    "provisioningModel": "STANDARD"
                }
            }

        ],
        "serviceAccount": {
            "email": "sa-batch@<PROJECT_ID>.iam.gserviceaccount.com"
        }
    },
    "logsPolicy": {
        "destination": "CLOUD_LOGGING"
    }
}

$ gcloud beta batch jobs submit \
      --location asia-northeast1 \
      --config ./job.json \
      --script-text "$(cat ./script.sh)"

$ gcloud storage ls gs://hanzawa-yuya-uncompressed-files/test/

> gs://hanzawa-yuya-uncompressed-files/test/sample.csv